
#!/usr/bin/python
# -*- coding: UTF-8 -*-

import os
import datetime
import platform
import sys
import shutil

import  hashlib
#from textparamsparse import *

"""
统计、备份文件
通过参数文件进程参数输入，参数文件如下：

sourcepath 要备份或统计的目录,可以重复
sourcepath=E:/data  
sourcepath=E:/data1

logpath :统计要输出的日志路径
logpath=E:/log

descpath 备份的目的路径
descpath=E:/wiredtirge

scantyp 对源路径进行扫描的方式， sha256 ：文件的256码，  filename :文件名
scantype=sha256
"""

"""
def statpathfileattr(paramfilename):  统计目录文件信息, paramfilename 参数文件

def backuppathfile(paramfilename): 备份文件 , paramfilename 参数文件
"""

"""
文本餐宿解析，参数按行存储，例：
同类参数可重复, #为注释
sourcepath=E:/data
sourcepath=E:/data

logpath=E:/wirged
"""

def parseparams(lines = None):
    """
    lines param 【“param”,"param2"】参数列表,
    """

    paramsdict = {}

    if lines == None:
        print("params lines is null")
        return None

    for line in lines:
        text = line.strip()
        if text == "" or len(text) == 0:
            continue
        if text[0] == "#":
            continue


        paramlist = text.split("=")
        if paramlist == None or len(paramlist) <= 1:
            continue

        if len(paramlist) == 2:
            key = paramlist[0]
            value = paramlist[1]

            key = key.strip()
            if len(key) == 0:
                continue
            value = value.strip()

            if len(value) == 0:
                continue

            valuelist = None
            """字典中已经存在的值"""
            existvalue = None

            try:
                existvalue = paramsdict[key]
            except Exception as e:
                existvalue = None

            if existvalue == None:
                valuelist = None
            else:
                if isinstance(existvalue, list):
                    valuelist = existvalue
                    valuelist.append(value)
                if isinstance(existvalue, str):
                    valuelist = []
                    valuelist.append(existvalue)
                    valuelist.append(value)
            if existvalue == None:
                paramsdict[key] = value
            if existvalue and valuelist:
                paramsdict[key] = valuelist

    return paramsdict

def getparamsfromparamdict(d, paramkey, paramtype="bool"):
    """
    从参数字典中获得参数
    d: 参数字典对象
    paramkey :参数关键字
    paramtype ： 参数类型，将根据类型进行转换
    """
    v = None

    try:
        v = d[paramkey]
    except Exception as e:
        print(e)
        return None

    if paramtype == "bool":
        if v.lower() == "true":
            return  True
        else:
            return False
    elif paramtype == "float":
        f = 0.0
        try:
            f = float(v)
        except Exception as e:
            print(e)
            f = None
        return f
    elif paramtype == "int":
        f = 0
        try:
            f = int(v)
        except Exception as e:
            print(e)
            f = None
        return f
    else:
        return v


def mkdir(path):
    # 引入模块
    import os

    # 去除首位空格
    path = path.strip()
    # 去除尾部 \ 符号
    #path = path.rstrip("\\")

    # 判断路径是否存在
    # 存在     True
    # 不存在   False
    isExists = os.path.exists(path)

    # 判断结果
    if not isExists:
        # 如果不存在则创建目录
        # 创建目录操作函数
        try:
            os.makedirs(path)
        except:
            return False

        print(path + " create successed!")
        return True
    else:
        # 如果目录存在则不创建，并提示目录已存在
        print(path + "is exist!")
        return True


def CalcFileSha256(filname):
    ''' calculate file sha256 '''
    bufsize = 1024 * 1024 * 16
    with open(filname, "rb") as f:
        sha256obj = hashlib.sha256()
        while True:
            data = f.read(bufsize)
            if data  == None or len(data) == 0:
                break
            sha256obj.update(data)


        hash_value = sha256obj.hexdigest()
        return hash_value


class CoSailTextFile():
    def __init__(self, filename = None):
        self.filename = filename
        self.f = None

    def open(self, m):
        try:
            self.f = open(self.filename, m)
        except Exception as e:
            print(e)
            self.f = None
            return False
        return True

    def writeLine(self, line):
        try:
            self.f.write(line + "\n")
            return True
        except Exception as  e:
            print(e)
            return False

    def writeLines(self, lines):
        for line in lines:
            if self.writeLine(line) == False:
                return False
        return True

    def readLine(self):
        text = None
        try:
             text = self.f.readline()
        except Exception as e:
            print(e)
            return None

        return text

    def readAll(self):
        return self.f.read()

    def readLines(self):
        return self.f.readlines()


    def close(self):
        if self.f != None:
            try:
                self.f.close()
                self.f = None
            except Exception as e:
                print(e)
                return False
            return True
        return True

    def isOpen(self):
        return self.f != None

def getFilePathExtend(fileName):
    filePath, suffix = os.path.splitext(fileName)
    filePath = os.path.dirname(fileName)
    l = len(suffix)
    fileName = os.path.split(fileName)[1]
    return filePath,  suffix[1:l + 1], fileName

filesdict = {}
def scansamenamefile(path, type= "filename", filesuffix = None):
    """
    扫描目录内重复文件，
    :param path: 扫描路径
    :param type: 相同文件判别方式，"filename" : 文件名  "sha256":sha256码
    filesuffix : 扫描指定后最的文件，None ：全扫描
    :return:
    """

    if os.path.exists(path) == False:
        print("Store Path :" + path + " isn't exist!")
        return None
    try:
        files = os.listdir(path)
    except FileNotFoundError:
        print("File Not FoundError")
        return None
    for fi in files:
        # file full path
        # sleep 1
        # time.sleep(1)
        fi_d = os.path.join(path, fi)
        samefilelist = None
        # is directory
        if os.path.isdir(fi_d):
            if os.path.islink(fi_d):
                continue
            """ recursion"""
            scansamenamefile(fi_d, type= type, filesuffix=filesuffix)

            # print("dir = ", fi_d)
        # is file
        else:
            file_path, suffix, file_name = getFilePathExtend(fi_d)

            if filesuffix != None:
                if suffix.strip().lower() != filesuffix.strip().lower():
                    continue

            tmpfullpath = file_path + "/" + file_name
            samekey = ""
            sha256 = CalcFileSha256(tmpfullpath)
            if type == "filename":
                samekey = file_name
            if type == "sha256":
                samekey = sha256
            try:
                samefilelist = filesdict[samekey]
            except Exception as e:
                print(e)
                samefilelist = None
            if samefilelist == None:
                samefilelist = []
                filesdict[samekey] = samefilelist

            fileattrdict = {}

            fileattrdict["filepath"] = tmpfullpath
            fileattrdict["size"] = os.path.getsize(tmpfullpath)
            fileattrdict["sha256"] = sha256#CalcFileSha256(tmpfullpath)
            fileattrdict["filename"] = file_name
            fileattrdict["suffix"] = suffix


            samefilelist.append(fileattrdict)
    return filesdict

def statallfiles(d):
    v = 0;
    o = None
    for k in d:
        o = d[k]
        v += len(o)
    return v

def statsamenamefiles(d):
    v = 0;
    o = None
    av = 0
    for k in d:
        o = d[k]
        if len(o) >=2:
            v += 1
            av += len(o)
    return v, av

def statallfilesbytype(d, suffix="sgy"):
    """
    那文件类型统计文件总数
    """
    v = 0;
    obj = None
    for k in d:
        obj = d[k]
        if len(obj) == 0:
            continue
        if obj[0]["suffix"] == suffix:
            v += len(obj)
    return v

def statsamefilesbytype(d, suffix = "sgy"):
    """
    按类型统计重复文件
    :param type:
    :return:
    """
    v = 0
    av = 0
    o = None
    for k in d:
        o = d[k]
        l = len(o)
        if l <=1:
            continue
        if o[0]["suffix"] == suffix:
            v += 1
            av += l

    return v, av


def outduplicatefilemsg(d, textFile):
    v = 0;
    o = None
    textFile.writeLine("")
    textFile.writeLine("**************Duplicate file list*****************")

    for k in d:
        o = d[k]
        if len(o) >= 2:
            textFile.writeLine("")
            txt = "file name : " + o[0]["filename"] + " sha256 : " +o[0]["sha256"] +  " Duplicate : " +str(len(o))
            textFile.writeLine(txt)

            txt = "file list :"
            for obj in o:
                txt ="fileputh :" + obj["filepath"] + " size : " + str(obj["size"]) + " SHA256 = " + obj["sha256"]
                textFile.writeLine(txt)

def outonefilemsg(d, textFile):
    v = 0;
    o = None
    textFile.writeLine("")
    textFile.writeLine("**************Unique file list*****************")

    for k in d:
        o = d[k]
        if len(o) == 1:

            for obj in o:
                txt ="fileputh :" + obj["filepath"] + " size : " + str(obj["size"])+ " SHA256 = " + obj["sha256"]
                textFile.writeLine(txt)

def statpathfileattr(paramsfilename):
    """
    :param pathlistfilename:  要统计的文件目录列表名，每行一个目录
    :param logpath 日志文件路径
    :return:
    """
    """
    
    """
    if os.path.exists(paramsfilename) == False:
        print("path list filename isn't exist!")
        return False


    f = CoSailTextFile(paramsfilename)
    f.open("r")
    lines = f.readLines()
    f.close()

    paramsdict = parseparams(lines)
    if paramsdict == None or len(paramsdict) == 0:
        print("params is null")
        return False

    sourcepath = None
    try:
        sourcepath = paramsdict["sourcepath"]
    except Exception as e:
        sourcepath = None

    scantype = None
    try:
        scantype = paramsdict["scantype"]
    except Exception as e:
        scantype = None

    logpath = None
    try:
        logpath = paramsdict["logpath"]
    except Exception as e:
        logpath = None

    if sourcepath == None:
        print("source path list is null")
        return False
    if logpath == None:
        print("log path is null ")
        return False

    if scantype == None:
        scantype = "filename"

    filesuffix = getparamsfromparamdict(paramsdict, "filesuffix", "str")

    if os.path.exists(logpath) == False:
        print("log path isn't exist")
        return

    if isinstance(sourcepath, str):
        filesdict.clear()
        statpathfileattrbypath(sourcepath, logpath, type = scantype, filesuffix=filesuffix)
        return True

    if isinstance(sourcepath, list) == False:
        return  False
    for line in sourcepath:
        line = line.strip('\n')
        filesdict.clear()
        statpathfileattrbypath(line, logpath, type = scantype, filesuffix=filesuffix)

    return True



def backuppathfile(paramfilename):
    """
    :param paramfilename:  备份文件列表中指定目录到目的目录
    :param descpath 备份目的mulu
    :return:
    """
    """

    """
    type = "filename"

    if os.path.exists(paramfilename) == False:
        print("path list filename isn't exist!")
        return


    f = CoSailTextFile(paramfilename)
    f.open("r")
    lines = f.readLines()
    f.close()


    if lines == None or len(lines) == 0:
        print("params is null")
        return False

    paramsdict = parseparams(lines)
    if paramsdict == None or len(paramsdict) == 0:
        print("params is null")
        return False

    isunique = True
    strtmp = "True"
    try:
        strtmp = paramsdict["isunique"]
    except Exception as e:
        isunique = True

    if strtmp.lower() == "true":
        isunique = True
    if strtmp.lower() == "false":
        isunique = False

    sourcepath = None
    try:
        sourcepath = paramsdict["sourcepath"]
    except Exception as e:
        sourcepath = None

    descpath = None
    try:
        descpath = paramsdict["descpath"]
    except Exception as e:
        descpath = None

    logpath = None

    scantype = None
    try:
        scantype = paramsdict["scantype"]
    except Exception as e:
        scantype = None

    try:
        logpath = paramsdict["logpath"]
    except Exception as e:
        logpath = None

    if sourcepath == None:
        print("source path list is null")
        return False

    if descpath == None:
        print("desc path list is null")
        return False

    if logpath == None :
        print("log path is null ")
        return False

    if scantype== None:
        scantype = "filename"

    #是否拷贝
    iscopy = getparamsfromparamdict(paramsdict, "iscopy", paramtype="bool")

    if iscopy == None:
        iscopy = True

    filesuffix = None

    filesuffix = getparamsfromparamdict(paramsdict, "filesuffix", "str")

    if os.path.exists(descpath) == False:
        print("backup descpath isn't exist")
        return

    if isinstance(sourcepath, str):
        path = sourcepath.strip('\n')
        filesdict.clear()
        sourcefiledict = scansamenamefile(path, type = scantype, filesuffix= filesuffix)
        if sourcefiledict == None:
            return False
        backupfiletodest(descpath, path, sourcefiledict = sourcefiledict, isunique=isunique, iscopy = iscopy)
        return True


    for path in sourcepath:
        path  = path .strip('\n')
        filesdict.clear()
        sourcefiledict = scansamenamefile(path, type = scantype, filesuffix= filesuffix)
        if sourcefiledict == None:
            continue
        backupfiletodest(descpath, path, sourcefiledict = sourcefiledict, isunique=isunique,  iscopy = iscopy)


def statpathfileattrbypath(path, logpath,  type= "filename", filesuffix = None):
    """
    统计指定路径的文件数量以及重复文件，并输出统计信息
    :param path: 要统计的路径
    :param logpath: 统计信息存储路基
    :return:
    """
    d = scansamenamefile(path, type= type, filesuffix= filesuffix)

    if d == None :
        return

    print("allfiles", statallfiles(d))

    #f.writeLine("all sgy files : " + str(statallfilesbytype(d, suffix="sgy")))
    #f.writeLine("all segy files : " + str(statallfilesbytype(d, suffix="segy")))

    print("sameallfiles", statsamenamefiles(d))
    t = path.replace("/", "-");
    t = t.replace("\\", "-");
    t = t.replace(":", "")
    if (platform.system() == 'Windows'):
        pass
    elif (platform.system()=='Linux'):
        t = t[1:]

    lonfilename = logpath + "/" + t + datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + ".log"
    lonfilename = lonfilename.replace("//", "/")
    f = CoSailTextFile(lonfilename)

    f.open("w")

    f.writeLine("stat path : " + path)
    f.writeLine("all files : " + str(statallfiles(d)))

    f.writeLine("all sgy files : " + str(statallfilesbytype(d, suffix="sgy")))
    f.writeLine("all segy files : " + str(statallfilesbytype(d, suffix="segy")))

    dupfiles, dupallfiles = statsamenamefiles(d)
    f.writeLine("Duplicate files : " +str(dupfiles)+ " File counts : " + str(dupallfiles))
    outduplicatefilemsg(d, f)

    outonefilemsg(d, f)

    f.close()

def backupfiletodest(destpath, sourcepath, sourcefiledict=None, isunique = True, iscopy = True):
    """
    从原路径中拷贝文件
    :param destpath: 备份目的目录
    :param sourcepath:要备份的目录
    :param filedict:要备份目录中文件字典
     isunique :True ： 相同文件只备份一份 False:全部备份
     iscopy : True 对源文件进线拷贝 False 对源文件进行移动
    :return:
    """
    if isunique == False:
        for key in sourcefiledict:
            obj = sourcefiledict[key]
            if len(obj) == 0:
                continue
            for  fileattr in obj:
                filefullpath = fileattr["filepath"]
                sha256 = fileattr["sha256"]

                tmp, _, filename = getFilePathExtend(filefullpath)
                # tmp = filefullpath.replace(sourcepath, "")
                if (platform.system() == 'Windows'):
                     tmp = tmp.replace(sourcepath[0:2], "")
                else:
                    tmp = tmp.replace(sourcepath, "")  #

                destfullpath = destpath + tmp  # os.path.join(destpath, tmp)
                destfullpath = destfullpath.replace("//", "/")
                destfilename =  os.path.join(destfullpath, filename)
                if os.path.exists(destfullpath) == False:
                    mkdir(destfullpath)

                if os.path.exists(destfilename):
                    """
                    如果目的文件已经存在，计算目的文件的SHA256码，
                    如果相同不做处理，如果不同，目的文件重命名
                    """
                    if CalcFileSha256(destfilename) == sha256:
                        print("source file : " + filefullpath + " target file : " + destfilename + " is same file!")
                        continue
                    else:
                        print("source file : " + filefullpath + " target file : " + destfilename + " isn't sanme file!" +
                              " rename " +destfilename +".1" )
                        for inx in range(1, 10000):
                            if os.path.exists(destfilename + "." + str(inx)) == True:
                                continue
                            destfilename = destfilename + "." + str(inx)
                            break

                if iscopy:
                    #拷贝问价

                    try:
                        shutil.copyfile(filefullpath, destfilename)
                    except Exception as e:
                        print(e)
                else:
                    #移动文件
                    try:
                        shutil.move(filefullpath, destfilename)
                    except Exception as e:
                        print(e)
        return True
    #备份一份
    else:
        for key in sourcefiledict:
            obj = sourcefiledict[key]
            if len(obj) == 0:
                continue
            fileattr = obj[0]
            filefullpath = fileattr["filepath"]

            tmp,_, filename = getFilePathExtend(filefullpath)

            #tmp = filefullpath.replace(sourcepath, "")

            if (platform.system() == 'Windows'):
                tmp = tmp.replace(sourcepath[0:2], "")
            else:
                tmp = tmp.replace(sourcepath, "")#

            destfullpath = destpath + tmp#os.path.join(destpath, tmp)
            destfullpath = destfullpath.replace("//", "/")
            if os.path.exists(destfullpath) == False:
                mkdir(destfullpath)
            try:
                shutil.copyfile(filefullpath, os.path.join(destfullpath, filename))
            except Exception as e:
                print(e)
        return True


if __name__ == '__main__':

    filesdict.clear()

    argv = sys.argv

    isstat = True

    for v in argv:
        print(v)

    paramfile = None

    if len(argv) == 1:
        filepath, _, _ = getFilePathExtend(argv[0])
        #默认参数文件 params.cfg
        paramfile = filepath + "/params.cfg"
        paramfile = paramfile.replace("//", "/")

        f = CoSailTextFile(paramfile)
        f.open("r")
        lines = f.readLines()
        f.close()

        paramsdict = parseparams(lines)
        isstat = getparamsfromparamdict(paramsdict, "isstat", "bool")
        #statpathfileattr(paramfile)
        if isstat:
            statpathfileattr(paramfile)
        else:
            backuppathfile(paramfile)
    if len(argv) >= 2:
        lines = argv[1:]

        paramsdict = parseparams(lines)
        isstat = getparamsfromparamdict(paramsdict, "isstat", "bool")
        paramfile = getparamsfromparamdict(paramsdict, "paramfile", "str")

        if isstat == None:
            isstat = True
        """
        参数argv[1]为参数文件路径
        """
        if isstat:
            statpathfileattr(paramfile)
        else:
            backuppathfile(paramfile)


    """
    if len(argv) == 2:
        statpathfileattr(argv[1])
    else:
        statpathfileattr("E:/1.txt")
    """
    """
    if (len(argv) == 3):
       statpathfileattr(argv[1], argv[2])
    elif (len(argv) == 2):
        statpathfileattr(argv[1], None)
    else:
        statpathfileattr("E:/1.txt", "E:/")
    """
    """
    "E:/1.txt" : 目录列表
    E:/wiredtirge :备份目的目录
    "sha256" : 文件拷贝类型，按sha256 码， 或"filename" 文件名
    """
    #backuppathfile("E:/1.txt")

    #backupfiletodest("E:/wiredtirge", "E:/data",filesdict )


    #d = buildfilenamedictformfilelist("/home/hadoop/tmp/dir-and-files.list")
    """存储目的路径根目录"""
    """
    destrootpath = "/home/hadoop/tmp4"
    path = "E:/data"
    d =  scansamenamefile(path)


    print(d)

    for k in d:
        print("key =", k)
        print("files = ", len(d[k]))

    print("allfiles", statallfiles(d))
    print("sameallfiles", statsamenamefiles(d))

    f = CoSailTextFile("e:/sss.log")

    f.open("w")

    f.writeLine("stat path : " + path)
    f.writeLine("all files : " + str(statallfiles(d)))
    f.writeLine("Duplicate files : " +str(statsamenamefiles(d)))
    outduplicatefilemsg(d, f)

    outonefilemsg(d, f)

    f.close()

    f = CoSailTextFile("E:/1.txt")
    f.open("r")
    text = f.readLines()

    print((text))
    """

